#!/usr/bin/groovy
@Library('test-shared-library') _

properties(
        [
                pipelineTriggers([cron('H 16 * * *')]),
                buildDiscarder(logRotator(numToKeepStr: '30'))
        ]
)

def readPropertiesFile(file) {
    def properties = [:]
    readFile(file).split("\n").each { line ->
        if (!line.startsWith("#")) {
            def splits = line.split("=")
            properties[splits[0]] = splits[1]
        }
    }
    return properties
}

def getDebugInfo() {
    sh """
        kubectl describe service sparkling-water-app
        kubectl get pods --all-namespaces 
        kubectl describe pods
        kubectl get events
        kubectl logs sparkling-water-app
    """
}

def runCommandWithK8sDebugInfo(GString script) {
    def result = sh(script: script, returnStatus: true)
    if (result > 0) {
        getDebugInfo()
    }
    return result
}

def runRCommandWithK8sDebugInfo(GString script) {
    def result = runCommandWithK8sDebugInfo(script)
    if (result > 0) {
        error("The script finished with the exit code ${result}.")
    }
}

static def getKubernetesSparkVersions(props) {
    def sparkVersions = props["supportedSparkVersions"].split(" ").toList()
    def boundaryVersion = props["kubernetesSupportSinceSpark"]
    def list = new ArrayList<String>()
    list.addAll(sparkVersions.subList(sparkVersions.indexOf(boundaryVersion), sparkVersions.size()))
    return list
}

static String getSparklingVersion(props, sparkMajorVersion) {
    return "${props['version'].replace("-SNAPSHOT", "")}-${sparkMajorVersion}"
}

String getSparkVersion(sparkMajorVersion) {
    def versionLine = readFile("gradle-spark${sparkMajorVersion}.properties").split("\n").find() { line -> line.startsWith('sparkVersion') }
    return versionLine.split("=")[1]
}

String getScalaBaseVersion(sparkMajorVersion) {
    def scalaVersionLine = readFile("gradle-spark${sparkMajorVersion}.properties").split("\n").find() { line -> line.startsWith('scalaVersion') }
    def scalaVersion = scalaVersionLine.split("=")[1]
    return scalaVersion.count(".") == 1 ? scalaVersion : scalaVersion.substring(0, scalaVersion.lastIndexOf('.'))
}

def buildSparklingWaterImage(String type, String sparkMajorVersion, String sparklingVersion) {
    sh """
        eval \$(minikube -p minikube-${sparkMajorVersion} docker-env)
        export H2O_HOME=${env.WORKSPACE}/h2o-3 
        export SPARK_LOCAL_HOSTNAME=localhost
        bash -x ./bin/build-kubernetes-images.sh $type
        docker tag sparkling-water-$type:${sparklingVersion} sparkling-water:$type-${sparklingVersion}
        docker rmi sparkling-water-$type:${sparklingVersion}
    """
}

def removeSparkImages(String sparkMajorVersion, String sparkVersion) {
    sh """
        eval \$(minikube -p minikube-${sparkMajorVersion} docker-env)
        docker rmi spark-r:${sparkVersion}
        docker rmi spark-py:${sparkVersion}
        docker rmi spark:${sparkVersion}
    """

}

String getH2OBranchMajorVersion() {
    def versionLine = readFile("h2o-3/gradle.properties").split("\n").find() { line -> line.startsWith('version') }
    return versionLine.split("=")[1]
}

static String getH2OBranchMajorName() {
    return "bleeding_edge"
}

static String getH2OBranchBuildVersion() {
    return "1-SNAPSHOT"
}

def getBuildAndTestStagesForSparkVersion(commons, props, sparkMajorVersion) {
    return {
        // ask for node in order to run this build branch in parallel on a separate node
        node("large") {
            ws("${env.WORKSPACE}-spark-${sparkMajorVersion}") {
                cleanWs()
                checkout scm

                withSparklingWaterMinikubeImage(commons) {

                    startMinikubeCluster(sparkMajorVersion)
                    getBuildStage(sparkMajorVersion, props)
                    grantClusterRights()

                    def sparkVersion = getSparkVersion(sparkMajorVersion)
                    def sparklingVersion = getSparklingVersion(props, sparkMajorVersion)
                    def master = "k8s://" + getMinikubeMaster()

                    // Spark home within sparkling_water_tests image the code runs currently in, for a current sparkVersion
                    def currentSparkHome = "/home/jenkins/spark-${sparkVersion}-bin"

                    // Spark home within a built sparkling-water:scala/python/r/external-backend-... image
                    def swImageSparkHome = "/opt/spark/"

                    //  To avoid problems with Spark host name resolution
                    def customEnv = [
                            "SPARK_LOCAL_HOSTNAME=localhost",
                    ]

                    withEnv(customEnv) {

                        try {
                            runScalaTests(sparkMajorVersion, master, sparklingVersion, currentSparkHome, swImageSparkHome)
                            runPythonTests(sparkMajorVersion, master, sparklingVersion, currentSparkHome, swImageSparkHome)
                            runRTests(sparkMajorVersion, sparkVersion, master, sparklingVersion, currentSparkHome)
                        } catch (Exception e) {
                            getDebugInfo()
                            stopMinikubeCluster(sparkMajorVersion)
                            throw e
                        }
                    }

                    stopMinikubeCluster(sparkMajorVersion)
                }
            }
        }
    }
}

def getBuildStage(sparkMajorVersion, props) {
    return stage("Build Images, Spark $sparkMajorVersion") {
        def sparklingVersion = getSparklingVersion(props, sparkMajorVersion)
        def sparkVersion = getSparkVersion(sparkMajorVersion)
        unstash "shared"
        sh """
            sed -i 's/^h2oMajorName=.*\$/h2oMajorName=${getH2OBranchMajorName()}/' gradle.properties
            sed -i 's/^h2oMajorVersion=.*\$/h2oMajorVersion=${getH2OBranchMajorVersion()}/' gradle.properties
            sed -i 's/^h2oBuild=.*\$/h2oBuild=${getH2OBranchBuildVersion()}/' gradle.properties
        """
        sh """
            export MASTER=localhost[*]
            export SPARK_LOCAL_HOSTNAME=localhost
            H2O_HOME=${env.WORKSPACE}/h2o-3 ./gradlew dist -Pspark=$sparkMajorVersion -Dmaven.repo.local=${env.WORKSPACE}/.m2 -PbuildAgainstH2OBranch=${props["testH2OBranch"]} -Ph2oMajorVersion=${getH2OBranchMajorVersion()} -Ph2oMajorName=${getH2OBranchMajorName()} -Ph2oBuild=${getH2OBranchBuildVersion()}
        """

        def customEnv = [
                "SPARK_HOME=/home/jenkins/spark-${sparkVersion}-bin",
        ]

        withEnv(customEnv) {
            dir("./dist/build/zip/sparkling-water-${sparklingVersion}") {
                buildSparklingWaterImage("scala", sparkMajorVersion, sparklingVersion)
                buildSparklingWaterImage("python", sparkMajorVersion, sparklingVersion)
                buildSparklingWaterImage("r", sparkMajorVersion, sparklingVersion)
                removeSparkImages(sparkMajorVersion, sparkVersion)
                buildSparklingWaterImage("external-backend", sparkMajorVersion, sparklingVersion)
            }
        }

        sh """
            eval \$(minikube -p minikube-${sparkMajorVersion} docker-env)
            docker image ls
        """
    }
}

def startMinikubeCluster(sparkMajorVersion) {
    return stage("Start Minikube, Spark $sparkMajorVersion") {
        // minikube default resources (2 CPU and 8GB) were not enough to run tests steadily
        // increasing available resources helped to stabilize the process
        sh """
            sudo chmod o+rw /var/run/docker.sock
            minikube delete -p minikube-${sparkMajorVersion} 
            minikube start -p minikube-${sparkMajorVersion}  --cpus 6 --memory 12g
        """
    }
}

def stopMinikubeCluster(sparkMajorVersion) {
    return stage("Stop Minikube, Spark $sparkMajorVersion") {
        sh "minikube stop -p minikube-${sparkMajorVersion}"
    }
}

def getBuildAndTestStages(commons, props) {
    def parallelStages = [:]
    getKubernetesSparkVersions(props).each { sparkMajorVersion ->
        parallelStages["Build & Test Spark ${sparkMajorVersion}"] = getBuildAndTestStagesForSparkVersion(commons, props, sparkMajorVersion)
    }
    return parallelStages
}

def testScalaInternalBackendClusterMode(master, version, currentSparkHome) {
    stage("Scala, Internal Backend, Cluster Mode") {
        sh "kubectl delete pod sparkling-water-app --ignore-not-found"
        runCommandWithK8sDebugInfo """
             ${currentSparkHome}/bin/spark-submit \
             --conf spark.kubernetes.container.image=sparkling-water:scala-${version} \
             --conf spark.driver.host=sparkling-water-app \
             --conf spark.kubernetes.driver.pod.name=sparkling-water-app \
             --conf spark.scheduler.minRegisteredResourcesRatio=1 \
             --master $master \
             --deploy-mode cluster \
             --name test \
             --class ai.h2o.sparkling.KubernetesTest \
             --conf spark.executor.instances=3 \
             local:///opt/sparkling-water/tests/kubernetesTest.jar
        """
        sh "sleep 60"
        sh "kubectl logs sparkling-water-app"
        sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
    }
}

def testPythonInternalBackendClusterMode(master, version, currentSparkHome) {
    stage("Python, Internal Backend, Cluster Mode") {
        sh "kubectl delete pod sparkling-water-app --ignore-not-found"
        runCommandWithK8sDebugInfo """
            ${currentSparkHome}/bin/spark-submit \
             --conf spark.kubernetes.container.image=sparkling-water:python-${version} \
             --conf spark.kubernetes.driver.pod.name=sparkling-water-app \
             --conf spark.driver.host=sparkling-water-app \
             --conf spark.scheduler.minRegisteredResourcesRatio=1 \
             --master $master \
             --deploy-mode cluster \
             --name test \
             --conf spark.executor.instances=3 \
             local:///opt/sparkling-water/tests/initTest.py
        """
        sh "sleep 60"
        sh "kubectl logs sparkling-water-app"
        sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
    }
}

def testRInternalBackend(master, sparklingVersion, sparkVersion, script) {
    stage("R, Internal Backend") {
        sh "kubectl delete pod sparkling-water-app --ignore-not-found"
        runRCommandWithK8sDebugInfo """
            export KUBERNETES_MASTER=$master
            export REGISTRY=
            export SW_VERSION=$sparklingVersion
            export SPARK_VERSION=$sparkVersion
            Rscript --default-packages=methods,utils $script
        """
        sh 'sleep 60'
        sh "kubectl logs sparkling-water-app | grep -q \"Open H2O Flow in browser\" && echo \"OK\" || exit 1"
        sh "kubectl logs sparkling-water-app | grep -qv \"ASSERTION ERROR\" && echo \"OK\" || exit 1"
    }
}

def setupREnvironment(version) {
    echo "Step setupREnvironment"
    sh """
        R -e 'install.packages("h2o-3/h2o-r/h2o_${getH2OBranchMajorVersion()}.99999.tar.gz", type="source", repos=NULL)'
        R -e 'install.packages("dist/build/zip/sparkling-water-${version}/rsparkling_${version}.tar.gz", type="source", repos=NULL)'
    """
}

def installSparkHeadlessService() {
    sh """
kubectl delete service sparkling-water-app --ignore-not-found
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: sparkling-water-app
spec:
  clusterIP: "None"
  selector:
    spark-driver-selector: sparkling-water-app
EOF
        """
}

def testScalaInternalBackendClientMode(master, version, swImageSparkHome) {
    stage("Scala, Internal Backend, Client Mode") {
        sh "kubectl delete pod sparkling-water-app --ignore-not-found"
        installSparkHeadlessService()
        def image = "sparkling-water:scala-${version}"
        runCommandWithK8sDebugInfo """
            kubectl run -n default -i --tty sparkling-water-app --restart=Never --labels spark-driver-selector=sparkling-water-app \
            --image=${image} -- \
              ${swImageSparkHome}/bin/spark-submit \
             --conf spark.scheduler.minRegisteredResourcesRatio=1 \
             --conf spark.kubernetes.container.image=${image}  \
             --master $master \
             --name test \
             --class ai.h2o.sparkling.KubernetesTest \
             --conf spark.driver.host=sparkling-water-app \
             --conf spark.kubernetes.driver.pod.name=sparkling-water-app \
             --deploy-mode client \
             --conf spark.executor.instances=3 \
             local:///opt/sparkling-water/tests/kubernetesTest.jar
        """
        sh "sleep 60"
        sh "kubectl logs sparkling-water-app"
        sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
    }
}

def testPythonInternalBackendClientMode(master, version, swImageSparkHome) {
    stage("Python, Internal Backend, Client Mode") {
        sh "kubectl delete pod sparkling-water-app --ignore-not-found"
        installSparkHeadlessService()
        def image = "sparkling-water:python-${version}"
        runCommandWithK8sDebugInfo """
            kubectl run -n default -i --tty sparkling-water-app --restart=Never --labels spark-driver-selector=sparkling-water-app \
            --image=${image} -- \
            ${swImageSparkHome}/bin/spark-submit \
            --conf spark.scheduler.minRegisteredResourcesRatio=1 \
            --conf spark.kubernetes.container.image=${image}  \
            --master $master \
            --name test \
            --conf spark.driver.host=sparkling-water-app \
            --conf spark.kubernetes.driver.pod.name=sparkling-water-app \
            --deploy-mode client \
            --conf spark.executor.instances=3 \
            local:///opt/sparkling-water/tests/initTest.py
        """
        sh "sleep 60"
        sh "kubectl logs sparkling-water-app"
        sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
    }
}

def installH2OHeadlessService() {
    sh "kubectl delete service h2o-service --ignore-not-found"
    sh """
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: h2o-service
spec:
  type: ClusterIP
  clusterIP: None
  selector:
    app: h2o-k8s
  ports:
  - protocol: TCP
    port: 54321
EOF
    """
}

def startExternalH2OBackend(version) {
    installH2OHeadlessService()
    sh "kubectl delete statefulsets h2o-stateful-set --ignore-not-found"
    sh """
cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: h2o-stateful-set
  namespace: default
spec:
  serviceName: h2o-service
  replicas: 2
  selector:
    matchLabels:
      app: h2o-k8s
  template:
    metadata:
      labels:
        app: h2o-k8s
    spec:
      terminationGracePeriodSeconds: 10
      containers:
        - name: h2o-k8s
          image: 'sparkling-water:external-backend-${version}'
          resources:
            requests:
              memory: "2Gi"
          ports:
            - containerPort: 54321
              protocol: TCP
          readinessProbe:
            httpGet:
              path: /kubernetes/isLeaderNode
              port: 8081
            initialDelaySeconds: 5
            periodSeconds: 5
            failureThreshold: 1
          env:
          - name: H2O_KUBERNETES_SERVICE_DNS
            value: h2o-service.default.svc.cluster.local
          - name: H2O_NODE_LOOKUP_TIMEOUT
            value: '180'
          - name: H2O_NODE_EXPECTED_COUNT
            value: '2'
          - name: H2O_KUBERNETES_API_PORT
            value: '8081'
EOF
    """
    sh 'sleep 60'
}

def stopExternalH2OBackend() {
    sh "kubectl delete statefulsets h2o-stateful-set --ignore-not-found"
    sh "kubectl delete service h2o-service --ignore-not-found"
}

def testRExternalBackendManual(master, sparklingVersion, sparkVersion, script) {
    stage("R, External Backend, Manual Mode") {
        startExternalH2OBackend(sparklingVersion)
        sh "kubectl delete pod sparkling-water-app --ignore-not-found"
        runRCommandWithK8sDebugInfo """
            export KUBERNETES_MASTER=$master
            export REGISTRY=
            export SW_VERSION=$sparklingVersion
            export SPARK_VERSION=$sparkVersion
            export EXTRA_OPTIONS="spark.ext.h2o.backend.cluster.mode=external spark.ext.h2o.external.start.mode=manual spark.ext.h2o.external.memory=2G spark.ext.h2o.cloud.representative=h2o-service.default.svc.cluster.local:54321 spark.ext.h2o.cloud.name=root"
            Rscript --default-packages=methods,utils $script
        """
        sh 'sleep 60'
        sh "kubectl logs sparkling-water-app | grep -q \"Open H2O Flow in browser\" && echo \"OK\" || exit 1"
        sh "kubectl logs sparkling-water-app | grep -qv \"ASSERTION ERROR\" && echo \"OK\" || exit 1"
        stopExternalH2OBackend()
    }
}

def testRExternalBackendAuto(master, sparklingVersion, sparkVersion, script) {
    stage("R, External Backend, Auto Mode") {
        sh """
        kubectl delete namespace h2o --ignore-not-found
        kubectl create namespace h2o
        """
        sh "kubectl delete pod sparkling-water-app --ignore-not-found"
        runRCommandWithK8sDebugInfo """
            export KUBERNETES_MASTER=$master
            export REGISTRY=
            export SW_VERSION=$sparklingVersion
            export SPARK_VERSION=$sparkVersion
            export EXTRA_OPTIONS="spark.ext.h2o.backend.cluster.mode=external spark.ext.h2o.external.k8s.namespace=h2o spark.ext.h2o.external.start.mode=auto spark.ext.h2o.external.cluster.size=2 spark.ext.h2o.external.k8s.docker.image=sparkling-water:external-backend-${sparklingVersion} spark.ext.h2o.external.auto.start.backend=kubernetes spark.ext.h2o.external.memory=2G"
            Rscript --default-packages=methods,utils $script
        """
        sh 'sleep 60'
        sh "kubectl logs sparkling-water-app | grep -q \"Open H2O Flow in browser\" && echo \"OK\" || exit 1"
        sh "kubectl logs sparkling-water-app | grep -qv \"ASSERTION ERROR\" && echo \"OK\" || exit 1"
        stopExternalH2OBackend()
    }
}

static String externalBackendManualSharedSubmitCmd(sparkHome, master, version, mode, language) {
    return """${sparkHome}/bin/spark-submit \
             --conf spark.kubernetes.container.image=sparkling-water:${language}-${version} \
             --conf spark.driver.host=sparkling-water-app \
             --conf spark.kubernetes.driver.pod.name=sparkling-water-app \
             --conf spark.scheduler.minRegisteredResourcesRatio=1 \
             --master $master \
             --deploy-mode $mode \
             --name test \
             --conf spark.ext.h2o.backend.cluster.mode=external \
             --conf spark.ext.h2o.external.start.mode=manual \
             --conf spark.ext.h2o.external.memory=2G \
             --conf spark.ext.h2o.cloud.representative=h2o-service.default.svc.cluster.local:54321 \
             --conf spark.ext.h2o.cloud.name=root \
             --conf spark.executor.instances=2 \
             --conf spark.ext.h2o.cloud.timeout=120000 \
            """
}

static String externalBackendAutoSharedSubmitCmd(sparkHome, master, version, mode, language) {
    return """${sparkHome}/bin/spark-submit \
             --conf spark.kubernetes.container.image=sparkling-water:${language}-${version} \
             --conf spark.driver.host=sparkling-water-app \
             --conf spark.kubernetes.driver.pod.name=sparkling-water-app \
             --conf spark.scheduler.minRegisteredResourcesRatio=1 \
             --master $master \
             --deploy-mode $mode \
             --name test \
             --conf spark.ext.h2o.backend.cluster.mode=external \
             --conf spark.ext.h2o.external.start.mode=auto \
             --conf spark.ext.h2o.external.auto.start.backend=kubernetes \
             --conf spark.ext.h2o.external.memory=2G \
             --conf spark.ext.h2o.external.k8s.namespace=h2o \
             --conf spark.ext.h2o.external.k8s.docker.image=sparkling-water:external-backend-${version} \
             --conf spark.executor.instances=2 \
             --conf spark.ext.h2o.external.cluster.size=2 \
             --conf spark.ext.h2o.cloud.timeout=120000 \
            """
}

static String scalaExternalBackendSubmitCmd(sparkHome, master, version, mode, startMode) {
    def prefix
    if (startMode == "manual") {
        prefix = externalBackendManualSharedSubmitCmd(sparkHome, master, version, mode, "scala")
    } else {
        prefix = externalBackendAutoSharedSubmitCmd(sparkHome, master, version, mode, "scala")
    }
    return """${prefix} \
              --class ai.h2o.sparkling.KubernetesTest \
              local:///opt/sparkling-water/tests/kubernetesTest.jar
           """
}

def testScalaExternalBackendManualClusterMode(master, version, currentSparkHome) {
    stage("Scala, External Backend, Manual Cluster Mode") {
        startExternalH2OBackend(version)
        sh "kubectl delete pod sparkling-water-app --ignore-not-found"
        runCommandWithK8sDebugInfo "${scalaExternalBackendSubmitCmd(currentSparkHome, master, version, "cluster", "manual")}"
        sh "kubectl logs sparkling-water-app"
        sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
        stopExternalH2OBackend()
    }
}

def testScalaExternalBackendManualClientMode(master, version, swImageSparkHome) {
    stage("Scala, External Backend, Manual Client Mode") {
        startExternalH2OBackend(version)
        sh "kubectl delete pod sparkling-water-app --ignore-not-found"
        installSparkHeadlessService()
        def image = "sparkling-water:scala-${version}"
        runCommandWithK8sDebugInfo """
            kubectl run -n default -i --tty sparkling-water-app --restart=Never --labels spark-driver-selector=sparkling-water-app \
            --image=${image} -- ${scalaExternalBackendSubmitCmd(swImageSparkHome, master, version, "client", "manual")}
        """
        sh "kubectl logs sparkling-water-app"
        sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
        stopExternalH2OBackend()
    }
}

def testScalaExternalBackendAutoClusterMode(master, version, currentSparkHome) {
    stage("Scala, External Backend, Auto Cluster Mode") {
        sh """
            kubectl delete namespace h2o --ignore-not-found
            kubectl create namespace h2o
        """
        sh "kubectl delete pod sparkling-water-app --ignore-not-found"
        runCommandWithK8sDebugInfo "${scalaExternalBackendSubmitCmd(currentSparkHome, master, version, "cluster", "auto")}"
        sh "kubectl logs sparkling-water-app"
        sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
    }
}

def testScalaExternalBackendAutoClientMode(master, version, swImageSparkHome) {
    stage("Scala, External Backend, Auto Client Mode") {
        sh """
            kubectl delete namespace h2o --ignore-not-found
            kubectl create namespace h2o
        """
        sh "kubectl delete pod sparkling-water-app --ignore-not-found"
        installSparkHeadlessService()
        def image = "sparkling-water:scala-${version}"
        runCommandWithK8sDebugInfo """
            kubectl run -n default -i --tty sparkling-water-app --restart=Never --labels spark-driver-selector=sparkling-water-app \
            --image=${image} -- ${scalaExternalBackendSubmitCmd(swImageSparkHome, master, version, "client", "auto")}
        """
        sh "sleep 420"
        sh "kubectl logs sparkling-water-app"
        sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
    }
}

static String pythonExternalBackendSubmitCmd(sparkHome, master, version, mode, startMode) {
    def prefix
    if (startMode == "manual") {
        prefix = externalBackendManualSharedSubmitCmd(sparkHome, master, version, mode, "python")
    } else {
        prefix = externalBackendAutoSharedSubmitCmd(sparkHome, master, version, mode, "python")
    }
    return """$prefix \
              local:///opt/sparkling-water/tests/initTest.py
            """
}


def testPythonExternalBackendManualClusterMode(master, version, currentSparkHome) {
    stage("Python, External Backend, Manual Cluster Mode") {
        startExternalH2OBackend(version)
        sh "kubectl delete pod sparkling-water-app --ignore-not-found"
        runCommandWithK8sDebugInfo "${pythonExternalBackendSubmitCmd(currentSparkHome, master, version, "cluster", "manual")}"
        sh "kubectl logs sparkling-water-app"
        sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
        stopExternalH2OBackend()
    }
}

def testPythonExternalBackendManualClientMode(master, version, swImageSparkHome) {
    stage("Python, External Backend, Manual Client Mode") {
        startExternalH2OBackend(version)
        sh "kubectl delete pod sparkling-water-app --ignore-not-found"
        installSparkHeadlessService()
        def image = "sparkling-water:python-${version}"
        runCommandWithK8sDebugInfo """
            kubectl run -n default -i --tty sparkling-water-app --restart=Never --labels spark-driver-selector=sparkling-water-app \
            --image=${image} -- ${pythonExternalBackendSubmitCmd(swImageSparkHome, master, version, "client", "manual")}
        """
        sh "kubectl logs sparkling-water-app"
        sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
        stopExternalH2OBackend()
    }
}

def testPythonExternalBackendAutoClusterMode(master, version,currentSparkHome) {
    stage("Python, External Backend, Auto Cluster Mode") {
        sh """
            kubectl delete namespace h2o --ignore-not-found
            kubectl create namespace h2o
        """
        sh "kubectl delete pod sparkling-water-app --ignore-not-found"
        runCommandWithK8sDebugInfo "${pythonExternalBackendSubmitCmd(currentSparkHome, master, version, "cluster", "auto")}"
        sh "kubectl logs sparkling-water-app"
        sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
    }
}

def testPythonExternalBackendAutoClientMode(master, version, swImageSparkHome) {
    stage("Python, External Backend, Auto Client Mode") {
        sh """
            kubectl delete namespace h2o --ignore-not-found
            kubectl create namespace h2o
        """
        sh "kubectl delete pod sparkling-water-app --ignore-not-found"
        installSparkHeadlessService()
        def image = "sparkling-water:python-${version}"
        runCommandWithK8sDebugInfo """
            kubectl run -n default -i --tty sparkling-water-app --restart=Never --labels spark-driver-selector=sparkling-water-app \
            --image=${image} -- ${pythonExternalBackendSubmitCmd(swImageSparkHome, master, version, "client", "auto")}
        """
        sh "sleep 420"
        sh "kubectl logs sparkling-water-app"
        sh "kubectl get pod sparkling-water-app | grep -q Completed && echo \"OK\" || exit 1"
    }
}

def withSparklingWaterMinikubeImage(commons, code) {
    def repoUrl = commons.getAWSDockerRepo()
    commons.withAWSDocker {
        def image = "${repoUrl}/opsh2oai/sparkling_water_tests:" + commons.getDockerImageVersion()
        def dockerOptions = "--init --privileged --network=host -v /var/run/docker.sock:/var/run/docker.sock"
        groovy.lang.Closure initCode = {
            sh "activate_java_8"
        }
        commons.withDocker(image, code, dockerOptions, initCode)
    }
}

def getMinikubeMaster() {
    return sh(
            script: "kubectl config view --minify -o jsonpath='{.clusters[].cluster.server}'",
            returnStdout: true
    ).trim()
}

def grantClusterRights() {
    stage("Allow Spark Create PODs") {
        sh """
            kubectl delete clusterrolebinding default --ignore-not-found
            kubectl create clusterrolebinding default --clusterrole=edit --serviceaccount=default:default --namespace=default                
        """
    }
}

def runRTests(sparkMajorVersion, sparkVersion, master, sparklingVersion, currentSparkHome) {
    return stage("Test R, Spark ${sparkMajorVersion}") {

        setupREnvironment(sparklingVersion)

        def maxDuration = 30
        def script = "kubernetes/src/r/initTest.R"
        def customEnv = [
                "SPARK_HOME=${currentSparkHome}",
                "SPARK_LOCAL_HOSTNAME=localhost"
        ]

        withEnv(customEnv) {
            timeout(time: maxDuration, unit: 'MINUTES') { testRInternalBackend(master, sparklingVersion, sparkVersion, script) }
            timeout(time: maxDuration, unit: 'MINUTES') { testRExternalBackendManual(master, sparklingVersion, sparkVersion, script) }
            timeout(time: maxDuration, unit: 'MINUTES') { testRExternalBackendAuto(master, sparklingVersion, sparkVersion, script) }
        }
    }
}

def runPythonTests(sparkMajorVersion, master, sparklingVersion, currentSparkHome, swImageSparkHome) {
    return stage("Test Python, Spark ${sparkMajorVersion}") {
        def maxDuration = 30
        timeout(time: maxDuration, unit: 'MINUTES') { testPythonInternalBackendClusterMode(master, sparklingVersion, currentSparkHome) }
        timeout(time: maxDuration, unit: 'MINUTES') { testPythonInternalBackendClientMode(master, sparklingVersion, swImageSparkHome) }
        timeout(time: maxDuration, unit: 'MINUTES') { testPythonExternalBackendManualClusterMode(master, sparklingVersion, currentSparkHome) }
        timeout(time: maxDuration, unit: 'MINUTES') { testPythonExternalBackendManualClientMode(master, sparklingVersion, swImageSparkHome) }
        timeout(time: maxDuration, unit: 'MINUTES') { testPythonExternalBackendAutoClusterMode(master, sparklingVersion, currentSparkHome) }
        timeout(time: maxDuration, unit: 'MINUTES') { testPythonExternalBackendAutoClientMode(master, sparklingVersion, swImageSparkHome) }
    }
}

def runScalaTests(sparkMajorVersion, master, sparklingVersion, currentSparkHome, swImageSparkHome) {
    return stage("Test Scala, Spark ${sparkMajorVersion}") {
        def maxDuration = 20
        timeout(time: maxDuration, unit: 'MINUTES') { testScalaInternalBackendClusterMode(master, sparklingVersion, currentSparkHome) }
        timeout(time: maxDuration, unit: 'MINUTES') { testScalaInternalBackendClientMode(master, sparklingVersion, swImageSparkHome) }
        timeout(time: maxDuration, unit: 'MINUTES') { testScalaExternalBackendManualClusterMode(master, sparklingVersion, currentSparkHome) }
        timeout(time: maxDuration, unit: 'MINUTES') { testScalaExternalBackendManualClientMode(master, sparklingVersion, swImageSparkHome) }
        timeout(time: maxDuration, unit: 'MINUTES') { testScalaExternalBackendAutoClusterMode(master, sparklingVersion, currentSparkHome) }
        timeout(time: maxDuration, unit: 'MINUTES') { testScalaExternalBackendAutoClientMode(master, sparklingVersion, swImageSparkHome) }
    }
}

node("large") {
    cleanWs()
    checkout scm
    def commons = load 'ci/commons.groovy'
    def props = readPropertiesFile("gradle.properties")

    withSparklingWaterMinikubeImage(commons) {

        stage("Build H2O") {
            retryWithDelay(3, 60, {
                sh "git clone https://github.com/h2oai/h2o-3.git"
            })
            retryWithDelay(5, 1, {
                sh """
                    cd h2o-3
                    git checkout ${props["testH2OBranch"]}
                    . /envs/h2o_env_python3.6/bin/activate
                    unset CI
                    ./gradlew build --parallel -x check -Duser.name=ec2-user
                    ./gradlew publishToMavenLocal --parallel -Dmaven.repo.local=${env.WORKSPACE}/.m2 -Duser.name=ec2-user -Dhttp.socketTimeout=600000 -Dhttp.connectionTimeout=600000
                    ./gradlew :h2o-r:buildPKG -Duser.name=ec2-user
                    cd ..
                """
            })
            stash name: "shared", excludes: "h2o-3/h2o-py/h2o/**/*.pyc, h2o-3/h2o-py/h2o/**/h2o.jar", includes: "h2o-3/build/h2o.jar, h2o-3/h2o-dist/buildinfo.json, h2o-3/gradle.properties, .m2/**, h2o-3/h2o-py/h2o/**, h2o-3/h2o-r/h2o_*.99999.tar.gz"
        }
    }

    parallel(getBuildAndTestStages(commons, props))
}
